#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import time
import sys
from pybloom_live import ScalableBloomFilter

reload(sys)
sys.setdefaultencoding('utf-8')


def generate_venom_name():
    '''生成venom数据文件名称'''
    timestamp = time.time()
    timeArray = time.localtime(timestamp)
    time_str = time.strftime("%Y-%m-%d", timeArray)
    timestamp_str = str(timestamp).split('.')[0]
    log_name = './data/1454.log.{}-{}'.format(time_str, timestamp_str)
    time.sleep(1)
    print(log_name)
    return log_name

# file_name = 'bidding_history'
file_name = '/mnt5/contact_team/wangshx/sx_s1_bidding算法提取/招投标算法提取的联系方式存量数据_2022-01-14_11_dc'
index = 0
mobile_bloom = ScalableBloomFilter(initial_capacity=10000000, error_rate=0.001, mode=2)  # 慢点但是省内存
mobile_count = 0
phone_bloom = ScalableBloomFilter(initial_capacity=10000000, error_rate=0.001, mode=2)  # 慢点但是省内存
phone_count = 0
email_bloom = ScalableBloomFilter(initial_capacity=10000000, error_rate=0.001, mode=2)  # 慢点但是省内存
email_count = 0
other_bloom = ScalableBloomFilter(initial_capacity=10000000, error_rate=0.001, mode=2)  # 慢点但是省内存
other_count = 0

with open(file_name,'r' ) as f ,open('bloom_length', 'w'  ) as log_f:
# with open(file_name,'r' ,encoding='utf-8') as f ,open('bloom_length', 'w' ,encoding='utf-8' ) as log_f:
    ls = []
    for line in f:
        line = line.strip()
        if line:
            ls.append(line)
            dic = json.loads(line)
            biddingWinningInfoList = dic.get('biddingWinningInfoList',[])
            biddingPurchasingInfoList = dic.get('biddingPurchasingInfoList',[])
            biddingAgentInfoList = dic.get('biddingAgentInfoList',[])
            biddingProjectContact = dic.get('biddingProjectContact',[])

            contact_ls = biddingWinningInfoList+biddingPurchasingInfoList+biddingAgentInfoList+biddingProjectContact
            if contact_ls:
                for tmp_contact_dic in contact_ls:
                    if isinstance(tmp_contact_dic.get('contact'),list):
                        for dic in  tmp_contact_dic.get('contact'):
                            contact = dic.get('contact', '')
                            contactType = dic.get('contactType', '')
                            _id = "{}_{}".format(contact, contactType)
                            if contactType == 0:
                                mobile_count += 1
                                mobile_bloom.add(_id)
                            elif contactType == 1:
                                phone_bloom.add(_id)
                                phone_count += 1
                            elif contactType == 2:
                                email_bloom.add(_id)
                                email_count += 1
                            elif isinstance(contactType, int):
                                other_bloom.add(_id)
                                other_count += 1
                    else:
                        contact = tmp_contact_dic.get('contact','')
                        contactType = tmp_contact_dic.get('contactType','')
                        _id = "{}_{}".format(contact,contactType)
                        if contactType == 0:
                            mobile_count += 1
                            mobile_bloom.add(_id)
                        elif contactType == 1:
                            phone_bloom.add(_id)
                            phone_count += 1
                        elif contactType == 2:
                            email_bloom.add(_id)
                            email_count += 1
                        elif isinstance(contactType,int):
                            other_bloom.add(_id)
                            other_count += 1


        if len(ls) == 50000:
            log_name = generate_venom_name()
            # with open(log_name,'w' ,encoding='utf-8') as f:
            with open(log_name,'w' ) as f:
                f.write('\n'.join(ls))
                ls = []
            print('当前bloom长度为：{}'.format(mobile_bloom.count))
            log_f.write('{}\t{}\t{}\t{}\n'.format(mobile_bloom.count,phone_bloom.count,email_bloom.count,other_bloom.count))
    else:
        if len(ls):
            log_name = generate_venom_name()
            # with open(log_name, 'w' ,encoding='utf-8') as f:
            with open(log_name, 'w' ) as f:
                f.write('\n'.join(ls))
            log_f.write(
                '{}\t{}\t{}\t{}\n'.format(mobile_bloom.count, phone_bloom.count, email_bloom.count, other_bloom.count))


